/**
 * 微博信息流采集
 * @type {import('playwright').Page}
 * @param {import('./types/collectors.js').ExtraData} extraData - 额外数据
 * @param {import('./types/collectors.js').CollectorOptions} options - 配置选项
 * 
 * @returns {import  } 采集
 */
async function scanAndFilterUrls(page, extraData, options) {
    const links = await page.locator('a[href*="//weibo.com"]').all();
    const filterList = ["https://weibo.com/u/" , 
                  "mygroup" , 
                  "https://weibo.com/aj/static/" ,
                  "https://weibo.com/tv",
                  "https://weibo.com/at/weibo",
                  "https://weibo.com/hot"  ];
    // 过滤函数 ，如果包含非目标的url 则返回false
    var filterFun = function (src){ return filterList.every((filterUrl)=>{ return src.indexOf(filterUrl) < 0 })  }              
    const hrefs = await Promise.all(
        links.map(async link => {
            const href = await link.getAttribute('href');
            return href;
        })
    );

    return hrefs.filter(href =>
        href && 
        !href.includes('https://weibo.com/u/') &&
        href.length > 'https://weibo.com/'.length * 1.2 && filterFun(href)
    ).map((url)=>{ return { title:"", url:url } });
}

export {
    scanAndFilterUrls
};